#!/usr/bin/env python3
# -*- coding: utf-8 -*-

# https://linux.cn/


import requests
import time
from bs4 import BeautifulSoup

from SpiderDataSink import *

#设置请求头部信息
headers = {'User-Agent':'Mozilla/5.0 (Windows NT 6.1) AppleWebKit/537.11 (KHTML, like Gecko) Chrome/23.0.1271.64 Safari/537.11',
    'Accept':'text/html;q=0.9,*/*;q=0.8',
    'Accept-Charset':'ISO-8859-1,utf-8;q=0.7,*;q=0.3',
    'Accept-Encoding':'gzip',
    'Connection':'close',
    'Referer':'http://www.baidu.com/link?url=_andhfsjjjKRgEWkj7i9cFmYYGsisrnm2A-TN3XZDQXxvGsM9k9ZZSnikW2Yds4s&amp;amp;wd=&amp;amp;eqid=c3435a7d00146bd600000003582bfd1f'
}

spider_url = "https://linux.cn/"
rq = requests.get(url=spider_url, headers=headers)
print("StatusCode:" , str(rq.status_code) , ", Encoding:" , rq.encoding)


#解析抓取的页面内容
sd = BeautifulSoup(rq.content,'html.parser')
blocks = sd.find_all('li')

def slim_text(inText):
    return inText.strip()

print("len(blocks):" + str(len(blocks)))

# anchors = [a for a in (td.find('a') for td in sd.findAll('span', attrs={'class':'story-title'})) if a]

for block in blocks:
    title = "ttt"
    content = "ccc"
    Category = "Miscellouse"
    link_url = "https://..."

    header_title_span = block.find('span', attrs={'class':'title'}) 
    if header_title_span: 
        title = slim_text(header_title_span.get_text())
        title = (title.encode('utf-8')).decode('utf-8')
        sourcelink = block.find("a")
        if sourcelink:
            link_url = sourcelink['href']
    else:
        continue

    ycat = block.find('span', attrs={'class': "y cat"})
    if ycat:
        Category = slim_text(ycat.get_text())
        Category = (Category.encode('utf-8')).decode('utf-8')

    body_div = block.find('p')
    if body_div:
        content = slim_text(body_div.get_text())

    print("=================================================")
    print("Category: " + Category)
    print("title: " + title)
    print("content: " + content)
    print("link_url: " + link_url)
    print("=================================================")

    dataSink({'category': Category, 'title': title, 'content': content, 'link_url': link_url, 'origin': 'linuxcn'})



